{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost Parameter Tuning for Rental Listing Inquiries Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第三步：调整树的参数：max_depth & min_child_weight\n",
    "(将步长降为1，进行精细调整)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先 import 必要的模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")\n",
    "#train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variable Identification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "选择该数据集的特征工程已经处理好了，集中精力放在参数调优上"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Target 分布，看看各类样本分布是否均衡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZYAAAEKCAYAAAAxXHOuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGodJREFUeJzt3X+wHWWd5/H3hwCCAibClY1JMKyb\nUSNqhLshyuygiBCYXQn+mIJ1JGVRG8cNiqVlCZYalcEf5aA1mRFmYpElTDnGiD+IbtgYEXF15UfA\nCITI5E4AiUmROEEIOqKBz/7Rz4WTcHJPJ+lzT87N51XVdfp8++k+365bybe6n6eflm0iIiKaclCv\nE4iIiLElhSUiIhqVwhIREY1KYYmIiEalsERERKNSWCIiolEpLBER0agUloiIaFQKS0RENOrgXifQ\nC8ccc4ynTp3a6zQiIvrGHXfc8WvbA3XaHpCFZerUqaxevbrXaURE9A1JD9Ztm1thERHRqK4WFkmH\nSbpN0s8lrZX0yRK/RtL9ktaUZUaJS9JCSUOS7pJ0Ysux5kpaX5a5LfGTJN1d9lkoSd08p4iIGFm3\nb4U9AZxm+3FJhwA/lnRD2fYh29ft0v4sYFpZTgauAk6W9AJgATAIGLhD0nLbj5Q284BbgBXAbOAG\nIiKiJ7p6xeLK4+XrIWUZaZ7+c4Bry363AOMlTQTOBFbZ3laKySpgdtl2lO2fupr//1pgTtdOKCIi\nOup6H4ukcZLWAFuoisOtZdPl5XbXFyU9p8QmAQ+17L6xxEaKb2wTb5fHPEmrJa3eunXrPp9XRES0\n1/XCYvtJ2zOAycBMSScAlwIvA/4z8ALgw6V5u/4R70W8XR6LbA/aHhwYqDViLiIi9sKojQqz/Rvg\nh8Bs25vL7a4ngP8FzCzNNgJTWnabDGzqEJ/cJh4RET3S7VFhA5LGl/XDgdOBX5S+EcoIrjnAPWWX\n5cAFZXTYLOBR25uBlcAZkiZImgCcAaws27ZLmlWOdQFwfTfPKSIiRtbtUWETgSWSxlEVsWW2vyvp\nB5IGqG5lrQH+qrRfAZwNDAG/A94FYHubpMuA20u7T9neVtbfA1wDHE41GiwjwiIiekjVYKoDy+Dg\noPPkfcT+7ZS/O6XXKYx5P3nvT2q3lXSH7cE6bfPkfURENCqFJSIiGpXCEhERjUphiYiIRqWwRERE\no1JYIiKiUSksERHRqBSWiIhoVApLREQ0KoUlIiIalcISERGNSmGJiIhGpbBERESjUlgiIqJRKSwR\nEdGoFJaIiGhUCktERDQqhSUiIhqVwhIREY1KYYmIiEalsERERKO6WlgkHSbpNkk/l7RW0idL/HhJ\nt0paL+lrkg4t8eeU70Nl+9SWY11a4vdJOrMlPrvEhiRd0s3ziYiIzrp9xfIEcJrtVwMzgNmSZgGf\nA75oexrwCHBhaX8h8Ijt/wR8sbRD0nTgPOAVwGzgSknjJI0DvgScBUwHzi9tIyKiR7paWFx5vHw9\npCwGTgOuK/ElwJyyfk75Ttn+Rkkq8aW2n7B9PzAEzCzLkO0Ntv8ALC1tIyKiR7rex1KuLNYAW4BV\nwL8Cv7G9ozTZCEwq65OAhwDK9keBo1vju+yzu3hERPRI1wuL7SdtzwAmU11hvLxds/Kp3Wzb0/iz\nSJonabWk1Vu3bu2ceERE7JVRGxVm+zfAD4FZwHhJB5dNk4FNZX0jMAWgbH8+sK01vss+u4u3+/1F\ntgdtDw4MDDRxShER0Ua3R4UNSBpf1g8HTgfWATcBbyvN5gLXl/Xl5Ttl+w9su8TPK6PGjgemAbcB\ntwPTyiizQ6k6+Jd385wiImJkB3dusk8mAkvK6K2DgGW2vyvpXmCppL8GfgZcXdpfDfyTpCGqK5Xz\nAGyvlbQMuBfYAcy3/SSApIuAlcA4YLHttV0+p4iIGEFXC4vtu4DXtIlvoOpv2TX+e+DtuznW5cDl\nbeIrgBX7nGxERDQiT95HRESjUlgiIqJRKSwREdGoFJaIiGhUCktERDQqhSUiIhqVwhIREY0a8TkW\nSR8YabvtLzSbTkRE9LtOD0geOSpZRETEmDFiYbH9ydFKJCIixoZafSyS/kTSjZLuKd9fJemj3U0t\nIiL6Ud3O+y8DlwJ/hKfnADuvW0lFRET/qltYnmv7tl1iO9q2jIiIA1rdwvJrSS+hvJ1R0tuAzV3L\nKiIi+lbdafPnA4uAl0n6FXA/8I6uZRUREX2rbmF50Pbpkp4HHGR7ezeTioiI/lX3Vtj9khZRva/+\n8S7mExERfa5uYXkp8H2qW2L3S/p7SX/avbQiIqJf1Sostv/d9jLbb6F61fBRwM1dzSwiIvpS7Uko\nJZ0q6UrgTuAw4C+6llVERPStWp33ku4H1gDLgA/Z/m1Xs4qIiL5V94rl1bbPtf3VPSkqkqZIuknS\nOklrJV1c4p+Q9CtJa8pydss+l0oaknSfpDNb4rNLbEjSJS3x4yXdKmm9pK9JOrRufhER0by6heU/\n7OVcYTuAD9p+OdWIsvmSppdtX7Q9oywrynGnU00V8wpgNnClpHGSxgFfAs4CpgPntxznc+VY04BH\ngAtrnlNERHRBV+cKs73Z9p1lfTuwDpg0wi7nAEttP2H7fmAImFmWIdsbbP8BWAqcI0nAacB1Zf8l\nwJya5xQREV0wanOFSZpKNaLs1hK6SNJdkhZLmlBik4CHWnbbWGK7ix8N/Mb2jl3iERHRI6MyV5ik\nI4BvAO+3/RhwFfASYEY5zhXDTdvs7r2It8thnqTVklZv3bq1buoREbGH9mWusL+ss6OkQ6iKylds\nfxPA9sMt278MfLd83QhMadl9MrCprLeL/xoYL+ngctXS2n4ntheVc2BwcLBt8YmIiH1X9wHJDbZP\nBwaAl9n+U9sPdNqv9IFcDayz/YWW+MSWZucC95T15cB5kp4j6XhgGnAbcDswrYwAO5Sqf2e5bQM3\nAW8r+88Frq9zThER0R0jXrFI+sBu4gC0FovdOAV4J3C3pDUl9hGqUV0zqG5bPQC8uxxvraRlwL1U\nfTjzbT9ZfvMiYCUwDlhse2053oeBpZL+GvgZVSGLiIge6XQr7Mh9ObjtH9O+H2TFCPtcDlzeJr6i\n3X62N1CNGouIiP3AiIXF9ifrHETSpbY/00xKERHRz2rPFdbB2xs6TkRE9LmmCku7210REXEAaqqw\nZPhuREQAuWKJiIiG1Soskk7pEPt6YxlFRERfq3vF8ncjxWx/upl0IiKi33V6QPK1wOuAgV0eljyK\n6kHFiIiInXR6QPJQ4IjSrvVhycd4ZhqViIiIp3V6QPJm4GZJ19h+EEDSQcARZZbiiIiIndTtY/mM\npKMkPY9qHq/7JH2oi3lFRESfqltYppcrlDlU83UdRzW5ZERExE7qFpZDyntV5gDX2/4jeSgyIiLa\nqFtY/pFqevvnAT+S9GKqDvyIiIid1HqDpO2FwMKW0IOS3tCdlCIiop/VffL+WElXS7qhfJ9O9bbG\niIiIndS9FXYN1dsbX1S+/wvw/m4kFBER/a1uYTnG9jLgKQDbO4Anu5ZVRET0rbqF5beSjqaMBJM0\nC3i0a1lFRETfqtV5D3wAWA68RNJPgAEypUtERLTRsbCUKVwOA04FXkr17pX7yrMsERERO+l4K8z2\nU8AVtnfYXmv7nrpFRdIUSTdJWidpraSLS/wFklZJWl8+J5S4JC2UNCTpLkknthxrbmm/XtLclvhJ\nku4u+yyUlJeORUT0UN0+lu9Jeute/Ke9A/ig7ZcDs4D5ZajyJcCNtqcBN5bvAGcB08oyD7gKqkIE\nLABOBmYCC4aLUWkzr2W/2XuYY0RENKhuYfkA1Vsi/yDpMUnbJXV88t72Ztt3lvXtwDpgEnAOsKQ0\nW0I1VQwlfq0rtwDjJU0EzgRW2d5m+xFgFTC7bDvK9k9tG7i25VgREdEDdZ+8P7Jzq5FJmgq8BrgV\nONb25nLszZJeWJpNAh5q2W1jiY0U39gmHhERPVL3yXtJ+ktJHyvfp0iaWfdHJB0BfAN4f4f3uLS7\n1ea9iLfLYZ6k1ZJWb926tVPKERGxl+reCrsSeC3w38v3x4Ev1dmxzIr8DeArtr9Zwg+X21iUzy0l\nvhGY0rL7ZGBTh/jkNvFnsb3I9qDtwYGBgTqpR0TEXqhbWE62PR/4PUDp5zi0006ls/9qYJ3tL7Rs\nWs4zc43NBa5viV9QrpBmAY+WW2YrgTMkTSid9mcAK8u27ZJmld+6oOVYERHRA3UfkPyjpHE88+T9\nAGV6lw5OoXoh2N2S1pTYR4DPAsskXQj8Enh72bYCOBsYAn4HvAvA9jZJlwG3l3afsr2trL+Hai6z\nw4EbyhIRET1St7AsBL4FvFDS5VRP3X+s0062f0z7fhCAN7Zpb2D+bo61GFjcJr4aOKFTLhERMTrq\njgr7iqQ7qIqBgDm213U1s4iI6Eu1Coukf7L9TuAXbWIRERFPq9t5/4rWL6W/5aTm04mIiH43YmGR\ndKmk7cCrWp643041PDijryIi4llGLCy2P1Oeuv+87aNsH1mWo21fOko5RkREH6nbeX+ppEnAi1v3\nsf2jbiUWERH9qW7n/WeB84B7eeaVxAZSWCIiYid1n2M5F3ip7Se6mUxERPS/uqPCNgCHdDORiIgY\nG+pesfwOWCPpRuDpqxbb7+tKVhER0bfqFpblZYmIiBhR3VFhSzq3ioiI6FBYJC2z/ReS7qbNC7Rs\nv6prmUVERF/qdMVycfn8r91OJCIixoYRC0vLe+kfHJ10IiKi39UdbhwREVFLCktERDSq0+zGN5bP\nz41OOhER0e86dd5PlHQq8GZJS9nlNcO27+xaZhER0Zc6FZaPA5cAk4Ev7LLNwGndSCoiIvpXp1Fh\n1wHXSfqY7ctGKaeIiOhjdZ+8v0zSm4E/K6Ef2v5u99KKiIh+VWtUmKTPUD0seW9ZLi6xTvstlrRF\n0j0tsU9I+pWkNWU5u2XbpZKGJN0n6cyW+OwSG5J0SUv8eEm3Slov6WuSDq132hER0S11hxv/OfAm\n24ttLwZml1gn15S2u/qi7RllWQEgaTrVy8ReUfa5UtI4SeOALwFnAdOB80tbgM+VY00DHgEurHk+\nERHRJXvyHMv4lvXn19mhvLp4W83jnwMstf2E7fuBIWBmWYZsb7D9B2ApcI4kUQ0euK7svwSYU/O3\nIiKiS+oWls8AP5N0jaQlwB3Ap/fhdy+SdFe5VTahxCYBD7W02Vhiu4sfDfzG9o5d4m1JmidptaTV\nW7du3YfUIyJiJLUKi+2vArOAb5bltbaX7uVvXgW8BJgBbAauKHG1aeu9iLdle5HtQduDAwMDe5Zx\nRETUVvdFX8MTUu7zy75sPzy8LunLwPDoso3AlJamk4FNZb1d/NfAeEkHl6uW1vYREdEjoz5XmKSJ\nLV/PBYZHjC0HzpP0HEnHA9OA24DbgWllBNihVB38y20buAl4W9l/LnD9aJxDRETsXu0rlr0h6avA\n64FjJG0EFgCvlzSD6rbVA8C7AWyvlbSMajjzDmC+7SfLcS4CVgLjgMW215af+DCwVNJfAz8Dru7m\n+URERGcdC4ukg4C7bJ+wpwe3fX6b8G7/87d9OXB5m/gKYEWb+AaqUWMREbGf6HgrzPZTwM8lHTcK\n+URERJ+reytsIrBW0m3Ab4eDtt/clawiIqJv1S0sn+xqFhERMWbUnYTyZkkvBqbZ/r6k51J1pEdE\nROyk7iSU/4Nq6pR/LKFJwLe7lVRERPSvus+xzAdOAR4DsL0eeGG3koqIiP5Vt7A8USaABEDSwYww\nfUpERBy46haWmyV9BDhc0puArwPf6V5aERHRr+oWlkuArcDdVE/KrwA+2q2kIiKif9UdFfZUmS7/\nVqpbYPeVuboiIiJ2UquwSPpz4B+Af6Warv54Se+2fUM3k4uIiP5T9wHJK4A32B4CkPQS4H8DKSwR\nEbGTun0sW4aLSrEB2NKFfCIios+NeMUi6S1lda2kFcAyqj6Wt1O9JyUiImInnW6F/beW9YeBU8v6\nVmDCs5tHRMSBbsTCYvtdo5VIRESMDXVHhR0PvBeY2rpPps2PiIhd1R0V9m2qNz9+B3iqe+lENOeX\nn3plr1M4IBz38bt7nULsZ+oWlt/bXtjVTCIiYkyoW1j+VtIC4HvAE8NB23d2JauIiOhbdQvLK4F3\nAqfxzK0wl+8RERFPq/uA5LnAf7R9qu03lKVjUZG0WNIWSfe0xF4gaZWk9eVzQolL0kJJQ5LuknRi\nyz5zS/v1kua2xE+SdHfZZ6Ek1T/1iIjohrqF5efA+L04/jXA7F1ilwA32p4G3Fi+A5wFTCvLPOAq\nqAoRsAA4GZgJLBguRqXNvJb9dv2tiIgYZXULy7HALyStlLR8eOm0k+0fAdt2CZ8DLCnrS4A5LfFr\nXbkFGC9pInAmsMr2NtuPAKuA2WXbUbZ/WmZavrblWBER0SN1+1gWNPibx9reDGB7s6ThVxxPAh5q\nabexxEaKb2wTb0vSPKqrG4477rh9PIWIiNiduu9jubnbiVBNx/+sn96LeFu2FwGLAAYHB/MumYiI\nLql1K0zSdkmPleX3kp6U9Nhe/ubD5TYW5XN4luSNwJSWdpOBTR3ik9vEIyKih2oVFttH2j6qLIcB\nbwX+fi9/czkwPLJrLnB9S/yCMjpsFvBouWW2EjhD0oTSaX8GsLJs2y5pVhkNdkHLsSIiokfq9rHs\nxPa3JV3SqZ2krwKvB46RtJGqr+azwDJJFwK/pJqCH2AFcDYwBPwOeFf5rW2SLuOZafo/ZXt4QMB7\nqEaeHU710rG8eCwiosfqTkL5lpavBwGDjNCfMcz2+bvZ9MY2bQ3M381xFgOL28RXAyd0yiMiIkZP\n3SuW1vey7AAeoBoeHBERsZO6o8LyXpaIiKil06uJPz7CZtu+rOF8IiKiz3W6Yvltm9jzgAuBo4EU\nloiI2EmnVxNfMbwu6UjgYqrRWkuBK3a3X0REHLg69rGUSSA/ALyDam6vE8ucXREREc/SqY/l88Bb\nqKZCeaXtx0clq4iI6Fudnrz/IPAi4KPAppZpXbbvw5QuERExhnXqY6k7rX5ERARQ/30sERERtaSw\nREREo1JYIiKiUSksERHRqBSWiIhoVApLREQ0KoUlIiIalcISERGNSmGJiIhGpbBERESjUlgiIqJR\nKSwREdGonhUWSQ9IulvSGkmrS+wFklZJWl8+J5S4JC2UNCTpLkknthxnbmm/XtLcXp1PRERUen3F\n8gbbM2wPlu+XADfangbcWL4DnAVMK8s84Cp4+iVkC4CTgZnAguFiFBERvdHrwrKrc6jeUkn5nNMS\nv9aVW4DxkiYCZwKrbG8rb7VcBcwe7aQjIuIZvSwsBr4n6Q5J80rsWNubAcrnC0t8EvBQy74bS2x3\n8YiI6JGO77zvolNsb5L0QmCVpF+M0FZtYh4h/uwDVMVrHsBxxx23p7lGRERNPbtisb2pfG4BvkXV\nR/JwucVF+dxSmm8EprTsPhnYNEK83e8tsj1oe3BgYKDJU4mIiBY9KSySnifpyOF14AzgHmA5MDyy\nay5wfVlfDlxQRofNAh4tt8pWAmdImlA67c8osYiI6JFe3Qo7FviWpOEc/tn2/5F0O7BM0oXAL4G3\nl/YrgLOBIeB3wLsAbG+TdBlwe2n3KdvbRu80IiJiVz0pLLY3AK9uE/834I1t4gbm7+ZYi4HFTecY\nERF7Z38bbhwREX0uhSUiIhrVy+HGfeGkD13b6xTGvDs+f0GvU4iIBuWKJSIiGpXCEhERjUphiYiI\nRqWwREREo1JYIiKiUSksERHRqBSWiIhoVApLREQ0KoUlIiIalcISERGNSmGJiIhGpbBERESjUlgi\nIqJRKSwREdGoFJaIiGhUCktERDQqhSUiIhqVwhIREY0aE4VF0mxJ90kaknRJr/OJiDiQ9X1hkTQO\n+BJwFjAdOF/S9N5mFRFx4Or7wgLMBIZsb7D9B2ApcE6Pc4qIOGCNhcIyCXio5fvGEouIiB44uNcJ\nNEBtYn5WI2keMK98fVzSfV3NqneOAX7d6yT2hP5mbq9T2J/03d+PBe3+CR6w+urvp/ft0d/uxXUb\njoXCshGY0vJ9MrBp10a2FwGLRiupXpG02vZgr/OIvZO/X3/L368yFm6F3Q5Mk3S8pEOB84DlPc4p\nIuKA1fdXLLZ3SLoIWAmMAxbbXtvjtCIiDlh9X1gAbK8AVvQ6j/3EmL/dN8bl79ff8vcDZD+rnzsi\nImKvjYU+loiI2I+ksIwhmdqmf0laLGmLpHt6nUvsGUlTJN0kaZ2ktZIu7nVOvZZbYWNEmdrmX4A3\nUQ3Bvh043/a9PU0sapH0Z8DjwLW2T+h1PlGfpInARNt3SjoSuAOYcyD/28sVy9iRqW36mO0fAdt6\nnUfsOdubbd9Z1rcD6zjAZ/9IYRk7MrVNRI9Jmgq8Bri1t5n0VgrL2FFrapuI6A5JRwDfAN5v+7Fe\n59NLKSxjR62pbSKieZIOoSoqX7H9zV7n02spLGNHpraJ6AFJAq4G1tn+Qq/z2R+ksIwRtncAw1Pb\nrAOWZWqb/iHpq8BPgZdK2ijpwl7nFLWdArwTOE3SmrKc3eukeinDjSMiolG5YomIiEalsERERKNS\nWCIiolEpLBER0agUloiIaFQKS0TDJI2X9D9H4XdeL+l13f6diD2VwhLRvPFA7cKiyt78W3w9kMIS\n+508xxLRMEnDM0vfB9wEvAqYABwCfNT29WWywhvK9tcCc4DTgQ9TTcWzHnjC9kWSBoB/AI4rP/F+\n4FfALcCTwFbgvbb/72icX0QnKSwRDStF47u2T5B0MPBc249JOoaqGEwDXgxsAF5n+xZJLwL+H3Ai\nsB34AfDzUlj+GbjS9o8lHQestP1ySZ8AHrf9N6N9jhEjObjXCUSMcQI+XV7k9RTVqwyOLdsetH1L\nWZ8J3Gx7G4CkrwN/UradDkyvpqQC4KjyQqmI/VIKS0R3vQMYAE6y/UdJDwCHlW2/bWnX7rUHww4C\nXmv731uDLYUmYr+SzvuI5m0Hhq8ong9sKUXlDVS3wNq5DThV0oRy++ytLdu+RzXBKACSZrT5nYj9\nRgpLRMNs/xvwE0n3ADOAQUmrqa5efrGbfX4FfJrqzYPfB+4FHi2b31eOcZeke4G/KvHvAOeW2XT/\nS9dOKGIPpfM+Yj8h6Qjbj5crlm8Bi21/q9d5ReypXLFE7D8+IWkNcA9wP/DtHucTsVdyxRIREY3K\nFUtERDQqhSUiIhqVwhIREY1KYYmIiEalsERERKNSWCIiolH/H0mmMnB4UroEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x83dfef0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(train.interest_level);\n",
    "pyplot.xlabel('target');\n",
    "pyplot.ylabel('Number of interest_level');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "每类样本分布不是很均匀，所以交叉验证时也考虑各类样本按比例抽取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# drop ids and get labels\n",
    "y_train = train['interest_level']\n",
    "\n",
    "train = train.drop([\"interest_level\"], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "各类样本不均衡，交叉验证是采用StratifiedKFold，在每折采样时各类样本按比例采样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第二轮参数调整得到的n_estimators最优值（219），max_depth为7，min_child_weight为5，其余参数继续默认值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "用交叉验证评价模型性能时，用scoring参数定义评价指标。评价指标是越高越好，因此用一些损失函数当评价指标时，需要再加负号，如neg_log_loss，neg_mean_squared_error 详见sklearn文档：http://scikit-learn.org/stable/modules/model_evaluation.html#log-loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [6, 7, 8], 'min_child_weight': [4, 5, 6]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = [6,7,8]\n",
    "min_child_weight = [4,5,6]\n",
    "param_test2_2 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58840, std: 0.00287, params: {'max_depth': 6, 'min_child_weight': 4},\n",
       "  mean: -0.58863, std: 0.00306, params: {'max_depth': 6, 'min_child_weight': 5},\n",
       "  mean: -0.58863, std: 0.00389, params: {'max_depth': 6, 'min_child_weight': 6},\n",
       "  mean: -0.58924, std: 0.00449, params: {'max_depth': 7, 'min_child_weight': 4},\n",
       "  mean: -0.58880, std: 0.00361, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.58992, std: 0.00375, params: {'max_depth': 7, 'min_child_weight': 6},\n",
       "  mean: -0.59380, std: 0.00471, params: {'max_depth': 8, 'min_child_weight': 4},\n",
       "  mean: -0.59243, std: 0.00381, params: {'max_depth': 8, 'min_child_weight': 5},\n",
       "  mean: -0.59200, std: 0.00446, params: {'max_depth': 8, 'min_child_weight': 6}],\n",
       " {'max_depth': 6, 'min_child_weight': 4},\n",
       " -0.58839841822716221)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=219,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=5,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_2 = GridSearchCV(xgb2_2, param_grid = param_test2_2, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_2.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_2.grid_scores_, gsearch2_2.best_params_,     gsearch2_2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 454.19079995,  456.84860001,  452.40079999,  503.95220003,\n",
       "         503.6178    ,  504.08999996,  559.89919996,  557.44739995,\n",
       "         521.08299994]),\n",
       " 'mean_score_time': array([ 1.27820005,  1.35639997,  1.2598    ,  1.66559997,  1.56860003,\n",
       "         1.54099998,  2.225     ,  2.0092    ,  1.68440003]),\n",
       " 'mean_test_score': array([-0.58839842, -0.58862871, -0.58863239, -0.58923501, -0.58879754,\n",
       "        -0.58991791, -0.59380009, -0.59243408, -0.59200358]),\n",
       " 'mean_train_score': array([-0.48268752, -0.48579975, -0.48892464, -0.43937799, -0.44481989,\n",
       "        -0.44937285, -0.39604813, -0.40428537, -0.41191113]),\n",
       " 'param_max_depth': masked_array(data = [6 6 6 7 7 7 8 8 8],\n",
       "              mask = [False False False False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'param_min_child_weight': masked_array(data = [4 5 6 4 5 6 4 5 6],\n",
       "              mask = [False False False False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'params': [{'max_depth': 6, 'min_child_weight': 4},\n",
       "  {'max_depth': 6, 'min_child_weight': 5},\n",
       "  {'max_depth': 6, 'min_child_weight': 6},\n",
       "  {'max_depth': 7, 'min_child_weight': 4},\n",
       "  {'max_depth': 7, 'min_child_weight': 5},\n",
       "  {'max_depth': 7, 'min_child_weight': 6},\n",
       "  {'max_depth': 8, 'min_child_weight': 4},\n",
       "  {'max_depth': 8, 'min_child_weight': 5},\n",
       "  {'max_depth': 8, 'min_child_weight': 6}],\n",
       " 'rank_test_score': array([1, 2, 3, 5, 4, 6, 9, 8, 7]),\n",
       " 'split0_test_score': array([-0.58380151, -0.58346045, -0.5817826 , -0.58227063, -0.58301462,\n",
       "        -0.58309901, -0.5861502 , -0.58547741, -0.58389605]),\n",
       " 'split0_train_score': array([-0.48390029, -0.48698385, -0.4900867 , -0.44065444, -0.4476738 ,\n",
       "        -0.45217294, -0.39662615, -0.40402716, -0.41244834]),\n",
       " 'split1_test_score': array([-0.58726781, -0.5884751 , -0.58797838, -0.58907479, -0.58712017,\n",
       "        -0.58964942, -0.59062045, -0.59117161, -0.5907653 ]),\n",
       " 'split1_train_score': array([-0.48165305, -0.48559293, -0.48917151, -0.43811254, -0.44358349,\n",
       "        -0.44926503, -0.39867053, -0.40515068, -0.41243351]),\n",
       " 'split2_test_score': array([-0.58905388, -0.58937444, -0.59015118, -0.58962561, -0.58981765,\n",
       "        -0.59167966, -0.59590752, -0.59554168, -0.59413821]),\n",
       " 'split2_train_score': array([-0.4817981 , -0.48501912, -0.49017183, -0.43764882, -0.44368944,\n",
       "        -0.44828809, -0.39678072, -0.40583275, -0.41098881]),\n",
       " 'split3_test_score': array([-0.58929665, -0.5887956 , -0.58961478, -0.58878071, -0.59015468,\n",
       "        -0.59080963, -0.59814019, -0.59507629, -0.59467867]),\n",
       " 'split3_train_score': array([-0.48338147, -0.48510524, -0.48721044, -0.43882942, -0.44348998,\n",
       "        -0.44811014, -0.39367747, -0.40431089, -0.41290559]),\n",
       " 'split4_test_score': array([-0.59257351, -0.59303929, -0.59363655, -0.59642549, -0.59388213,\n",
       "        -0.59435318, -0.59818342, -0.59490414, -0.59654104]),\n",
       " 'split4_train_score': array([-0.48270468, -0.48629761, -0.48798272, -0.44164475, -0.44566274,\n",
       "        -0.44902805, -0.39448578, -0.40210536, -0.41077939]),\n",
       " 'std_fit_time': array([ 12.18449055,   8.94154221,   4.6304671 ,  11.20704104,\n",
       "         11.399301  ,  12.09496771,  13.45838123,   7.65489113,  64.64655929]),\n",
       " 'std_score_time': array([ 0.03024173,  0.11165768,  0.06276108,  0.21352157,  0.02336331,\n",
       "         0.06489372,  0.23485566,  0.03793888,  0.33204673]),\n",
       " 'std_test_score': array([ 0.00286618,  0.00305838,  0.00388993,  0.0044851 ,  0.0036052 ,\n",
       "         0.00374535,  0.0047143 ,  0.00381395,  0.00446243]),\n",
       " 'std_train_score': array([ 0.00087337,  0.00074625,  0.00116555,  0.00152708,  0.00163893,\n",
       "         0.00146575,  0.00177813,  0.00126198,  0.00085813])}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_2.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'gsearch1' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-9-56d2a886c88b>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# summarize results\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[1;32mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Best: %f using %s\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mgsearch1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbest_score_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgsearch1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbest_params_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mtest_means\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgsearch2_2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcv_results_\u001b[0m\u001b[1;33m[\u001b[0m \u001b[1;34m'mean_test_score'\u001b[0m \u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mtest_stds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgsearch2_2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcv_results_\u001b[0m\u001b[1;33m[\u001b[0m \u001b[1;34m'std_test_score'\u001b[0m \u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mtrain_means\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgsearch2_2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcv_results_\u001b[0m\u001b[1;33m[\u001b[0m \u001b[1;34m'mean_train_score'\u001b[0m \u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'gsearch1' is not defined"
     ]
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_2.best_score_, gsearch2_2.best_params_))\n",
    "test_means = gsearch2_2.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_2.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_2.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_2.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_2.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_2.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(min_child_weight), len(max_depth))\n",
    "train_scores = np.array(train_means).reshape(len(min_child_weight), len(max_depth))\n",
    "\n",
    "for i, value in enumerate(min_child_weight):\n",
    "    pyplot.plot(max_depth, test_scores[i], label= 'test_min_child_weight:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( '- Log Loss' )\n",
    "pyplot.savefig( 'max_depth_vs_min_child_weght2.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
